import requests
from bs4 import BeautifulSoup
import time
import random
from datetime import datetime
import re
import json
import os
import glob
from collections import Counter


# 配置爬虫参数
BASE_URL = "http://www.jjwxc.net/"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
    "Referer": BASE_URL
}
REQUEST_DELAY = random.uniform(1, 3)  # 随机延迟

# 创建数据存储目录
os.makedirs("data", exist_ok=True)
covers_path = "covers"
os.makedirs(covers_path, exist_ok=True)


class JJWXCSpider:
    def __init__(self):

        # 定义要爬取的榜单URL
        self.rank_urls = {
            "季度排行榜": "https://www.jjwxc.net/topten.php?t=6&orderstr=4",
            "勤奋指数榜": "https://www.jjwxc.net/topten.php?t=6&orderstr=15",
            "八仙自然榜": {
                "重生": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10901",
                "幻想未来": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10905",
                "架空历史": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10903",
                "都市情缘": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10906",
                "穿越时空": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10904",
                "天赐良缘": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10907",
                "时尚娱乐": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10908",
                "奇幻异闻": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10902",
                "系统快穿": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10911",
                "宫廷侯爵": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10909",
                "业界精英": "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10910",
                '情有独钟': "https://www.jjwxc.net/channeltoplist.php?rchannelid=109&rankgroup=3&subchannelid=10912",
            }
        }

        self.session = requests.Session()
        self.session.headers.update(HEADERS)
        self.novels_data = []
        self.genre_mapping = {}         # 小说ID -> 八仙自然榜题材
        self.diligent_authors = []      # 勤奋作者ID

    def get_page(self, url):
        """获取页面内容"""
        time.sleep(REQUEST_DELAY)
        try:
            response = self.session.get(url)
            response.raise_for_status()
            response.encoding = "gbk"
            return response.text
        except Exception as e:
            print(f"请求失败: {url}, 错误: {e}")
            return None

    def parse_gl_channel(self):
        """解析百合频道榜单数据"""
        print("开始爬取晋江文学城百合频道榜单数据...")

        # 爬取季度排行榜（核心分析目标）
        print(f"爬取季度排行榜...")
        quarter_url = self.rank_urls["季度排行榜"]
        self.parse_quarter_rank(quarter_url)

        # 爬取八仙自然榜各子榜单（用于题材分类）
        print("爬取八仙自然榜...")
        baxian_ranks = self.rank_urls["八仙自然榜"]
        for sub_rank, url in baxian_ranks.items():
            print(f"爬取八仙自然榜: {sub_rank}")
            self.parse_baxian_for_classification(url, sub_rank)

        # 爬取勤奋指数榜（用于作者分析）
        print("爬取勤奋指数榜...")
        diligent_url = self.rank_urls["勤奋指数榜"]
        self.parse_diligent_rank(diligent_url)

        print(f"\n爬取完成! 共获取 {len(self.novels_data)} 部小说数据")
        self.save_data()
        return self.novels_data

    def parse_baxian_for_classification(self, url, sub_rank):
        """解析八仙自然榜用于题材分类"""
        html = self.get_page(url)
        if not html:
            print(f"八仙自然榜获取失败: {url}")
            return

        soup = BeautifulSoup(html, 'html.parser')
        # Select all rows except the header
        novels = soup.select("tbody tr")[1:]
        for novel in novels:
            try:
                tds = novel.select('td')
                title_elem = tds[2].select_one('a')
                novel_id = title_elem['href'].split('=')[-1]

                # 添加到分类映射
                self.genre_mapping[novel_id] = sub_rank
            except Exception as e:
                print(f"解析八仙自然榜小说失败: {e}")

    def parse_diligent_rank(self, url):
        """解析勤奋指数榜用于作者分析"""
        html = self.get_page(url)
        if not html:
            print(f"勤奋指数榜获取失败: {url}")
            return

        soup = BeautifulSoup(html, 'html.parser')
        # Select all rows except the header
        novels = soup.select("tbody tr")[1:]

        for novel in novels:
            try:
                # 只提取作者信息
                tds = novel.select('td')
                author_elem = tds[1].select_one('a')
                author_id = re.search(
                    r'authorid=(\d+)', author_elem['href']).group(1) if author_elem else ""

                # 标记为勤奋作者
                if author_id:
                    self.diligent_authors.append(author_id)

            except Exception as e:
                print(f"解析勤奋指数榜失败: {e}")

    def parse_quarter_rank(self, url):
        """深度解析季度排行榜"""
        html = self.get_page(url)
        if not html:
            print(f"季度排行榜获取失败: {url}")
            return

        soup = BeautifulSoup(html, 'html.parser')

        # Select all rows except the header
        novels = soup.select("tbody tr")[1:]

        for novel in novels:
            try:
                # 解析基础信息
                tds = novel.select('td')

                # 排名
                rank = int(tds[0].text.strip())

                # 作者
                author_elem = tds[1].select_one('a')
                author = author_elem.text.strip() if author_elem else "未知作者"
                author_url = BASE_URL + \
                    author_elem['href'] if author_elem else ""
                author_id = re.search(
                    r'authorid=(\d+)', author_url).group(1) if author_elem else ""

                # 标题和链接
                title_elem = tds[2].select_one('a')
                title = title_elem.text.strip()
                novel_url = BASE_URL + title_elem['href']
                novel_id = re.search(
                    r'onebook\.php\?novelid=(\d+)', novel_url).group(1)
                print(f"爬取小说 {title} 中...")

                # 文章类型
                type_text = tds[3].text.strip()
                # 解析文章类型
                type_parts = type_text.split('-')
                originality = type_parts[0] if len(type_parts) > 0 else "未知"
                orientation = type_parts[1] if len(type_parts) > 1 else "未知"
                era = type_parts[2] if len(type_parts) > 2 else "未知"
                genre = type_parts[3] if len(type_parts) > 3 else "未知"

                # 进度
                status = tds[4].text.strip()

                # 字数
                word_count_text = tds[5].text.strip()
                word_count = int(word_count_text)

                # 发表时间转换为datetime对象
                publish_time_text = tds[7].text.strip()
                publish_time = datetime.strptime(
                    publish_time_text, '%Y-%m-%d %H:%M:%S')

                # 访问详情页获取更多数据
                detail_data = self.get_novel_details(novel_url)

                novel_data = {
                    'novel_id': novel_id,
                    'title': title,
                    'novel_url': novel_url,
                    'author_id': author_id,
                    'author': author,
                    'author_url': author_url,
                    'word_count': word_count,
                    'status': status,
                    'rank_position': rank,
                    'type_text': type_text,
                    'originality': originality,
                    'orientation': orientation,
                    'era': era,
                    'genre': genre,
                    'cover_url': detail_data.get('cover_url', ""),
                    'perspective': detail_data.get('perspective', ""),
                    'contract_status': detail_data.get('contract_status', ""),
                    'intro': detail_data.get('intro', ""),
                    'tags': detail_data.get('tags', []),
                    'characters': detail_data.get('characters', []),
                    'other': detail_data.get('other', ""),
                    'brief': detail_data.get('brief', ""),
                    'themes': detail_data.get('themes', ""),
                    'chapters': detail_data.get('chapters', []),
                    # 'non_v_avg_clicks': detail_data.get('non_v_avg_clicks', 0),
                    # 'total_reviews': detail_data.get('total_reviews', 0),
                    # 'collected_count': detail_data.get('collected_count', 0),
                    # 'nutrition_count': detail_data.get('nutrition_count', 0),
                    # 'article_score': detail_data.get('article_score', 0),
                }

                # 下载封面
                self.download_cover(novel_data)

                self.novels_data.append(novel_data)

            except Exception as e:
                print(f"解析季度排行榜小说失败: {e}")

    def get_novel_details(self, novel_url):
        """获取小说详情信息"""
        html = self.get_page(novel_url)
        if not html:
            return {}

        try:
            soup = BeautifulSoup(html, 'html.parser')
            detail_data = {}

            # 获取封面图片URL
            cover_img = soup.select_one('img.noveldefaultimage')
            if cover_img:
                cover_url = cover_img.get('src')
                if cover_url:
                    if not cover_url.startswith('https://i9-static.jjwxc.net/novelimage.php'):
                        detail_data['cover_url'] = cover_url
                else:
                    detail_data['cover_url'] = ""
            else:
                detail_data['cover_url'] = ""

            # 定位关键信息容器
            info_ul = soup.select_one('ul.rightul[name="printright"]')
            if info_ul:
                # 解析所有信息项
                for li in info_ul.select('li'):
                    # 获取字段名称
                    field_name = li.select_one('span:first-child')
                    if not field_name:
                        continue

                    field_name = field_name.text.strip().replace('：', '')

                    # 存储关键字段
                    if field_name == "作品视角":
                        full_text = li.get_text(strip=True)
                        detail_data['perspective'] = full_text.replace(
                            field_name, '', 1).replace('：', '').strip()
                    elif field_name == "签约状态":
                        detail_data['contract_status'] = li.select_one(
                            'font').text.strip()

            # 获取文案
            intro_elem = soup.select_one('div#novelintro')
            detail_data['intro'] = intro_elem.get_text(
                separator="\n").strip() if intro_elem else ""

            # 获取内容标签
            detail_data['tags'] = [
                a.text for a in soup.select('div.smallreadbody span > a[style*="red"]')
            ]

            # 主角
            characters = soup.find_all('div', class_='character_name')
            detail_data['characters'] = [c.text.strip()
                                         for c in characters if c.text.strip()]

            # 其它
            other_elem = soup.select_one('span.bluetext')
            detail_data['other'] = other_elem.text.split(
                '：', 1)[-1].strip() if other_elem else ""

            # 立意
            themes_span = soup.find('span', string=lambda t: t and '立意' in t)
            if themes_span:
                detail_data['themes'] = themes_span.text.split(
                    '：', 1)[-1].strip()

            # 一句话简介
            brief_span = soup.find(
                'span', string=lambda text: text and '一句话简介' in text)
            if brief_span:
                # Extract text after "一句话简介："
                detail_data["brief"] = brief_span.text.split(
                    '：', 1)[-1].strip()

            # 获取章节信息
            detail_data["chapters"] = []
            chapter_elems = soup.select('tr[itemprop~="chapter"]')
            for row in chapter_elems:
                cols = row.find_all("td")

                is_vip, is_latest, is_unavailable = False, False, False

                update_time = cols[-1].find("span").text.strip()
                # 提取章节首发时间
                title_attr = cols[-1].get('title', '')
                pattern = r"章节首发时间：(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
                match = re.search(pattern, title_attr)
                publish_time = match.group(1).strip() if match else ""

                # Check if this is the latest chapter
                if "newestChapter" in row["itemprop"]:
                    is_latest = True

                if len(cols) < 6:
                    is_vip = True  # VIP chapter

                # 尝试获取链接
                link_element = cols[1].find("a", itemprop="url")
                if link_element and link_element.has_attr('href'):
                    url = link_element["href"]
                else:
                    # 等待进入网审 chapter 没有链接
                    is_unavailable = True
                    url = ""

                chapter = {
                    "number": cols[0].text.strip(),
                    "title": cols[1].get_text(separator=" ", strip=True),
                    "url": url,
                    "preview": cols[2].text.strip(),
                    "word_count": cols[3].text.strip(),
                    "publish_time": publish_time,
                    "update_time": update_time,
                    "is_latest": is_latest,
                    "is_vip": is_vip,
                    "is_unavailable": is_unavailable
                }

                detail_data["chapters"].append(chapter)

            # # 非v章节章均点击数
            # non_v_span = soup.select_one('span[itemprop="totalClick"]')
            # if non_v_span:
            #     detail_data['non_v_avg_clicks'] = int(non_v_span.text.strip())

            # # 总书评数
            # reviews_span = soup.select_one('span[itemprop="reviewCount"]')
            # if reviews_span:
            #     detail_data['total_reviews'] = int(reviews_span.text.strip())

            # # 当前被收藏数
            # collected_span = soup.select_one('span[itemprop="collectedCount"]')
            # if collected_span:
            #     detail_data['collected_count'] = int(
            #         collected_span.text.strip())

            # # 营养液数
            # nutrition_span = soup.select_one('span[itemprop="nutritionCount"]')
            # if nutrition_span:
            #     detail_data['nutrition_count'] = int(
            #         nutrition_span.text.strip())

            # # 文章积分
            # score_span = soup.select_one('span[itemprop="scoreCount"]')
            # if score_span:
            #     score_str = score_span.text.strip().replace(',', '')
            #     detail_data['article_score'] = int(score_str)

            return detail_data
        except Exception as e:
            print(f"获取小说详情失败: {e}")
            return {}

    def download_cover(self, novel):
        """下载小说的封面图片"""

        if novel['cover_url']:
            try:
                # 生成唯一文件名
                filename = f"{covers_path}/{novel['novel_id']}.jpg"

                # 如果文件已存在，跳过下载
                if os.path.exists(filename):
                    return

                # 下载图片
                response = self.session.get(
                    novel['cover_url'], stream=True, timeout=10)
                response.raise_for_status()

                # 保存图片
                with open(filename, 'wb') as f:
                    for chunk in response.iter_content(chunk_size=8192):
                        f.write(chunk)

            except Exception as e:
                print(f"下载封面失败: {novel['title']}, 错误: {e}")

    def save_data(self):
        """保存所有数据到文件"""
        # 保存小说数据
        with open('data/novels_data.json', 'w', encoding='utf-8') as f:
            json.dump(self.novels_data, f, ensure_ascii=False)

        # 保存题材映射
        with open('data/genre_mapping.json', 'w', encoding='utf-8') as f:
            json.dump(self.genre_mapping, f, ensure_ascii=False)

        # 保存勤奋作者列表
        with open('data/diligent_authors.json', 'w', encoding='utf-8') as f:
            json.dump(list(self.diligent_authors), f, ensure_ascii=False)

        print("数据已保存到 data 目录")

    def load_data(self):
        """从文件加载所有数据"""
        try:
            # 加载小说数据
            if os.path.exists('data/novels_data.json'):
                with open('data/novels_data.json', 'r', encoding='utf-8') as f:
                    self.novels_data = json.load(f)
            else:
                return False

            # 加载题材映射
            if os.path.exists('data/genre_mapping.json'):
                with open('data/genre_mapping.json', 'r', encoding='utf-8') as f:
                    self.genre_mapping = json.load(f)
            else:
                return False

            # 加载勤奋作者列表
            if os.path.exists('data/diligent_authors.json'):
                with open('data/diligent_authors.json', 'r', encoding='utf-8') as f:
                    self.diligent_authors = json.load(f)
            else:
                return False

            print("所有数据加载成功")
            return True
        except:
            print("没有找到数据文件，请先运行爬虫")
            return False


def create_cover_collage():
    """创建封面拼贴图"""
    print("开始创建封面拼贴图...")

    # 获取所有封面图片
    cover_files = glob.glob("covers/*.jpg")
    if not cover_files:
        print("没有可用的封面图片")
        return False

    # 随机抽取10张封面图片
    cover_files = random.sample(cover_files, 10)

    os.system(f"sh collage.sh {' '.join(cover_files)}")


class DataAnalyzer:
    def __init__(self, novels_data, genre_mapping, diligent_authors):
        self.novels_data = novels_data
        self.total_novels = len(novels_data)  # 小说总数
        self.genre_mapping = genre_mapping
        self.diligent_authors = diligent_authors

        self.genre_rules = {
            '种田文': {'keywords': ['种田文']},
            '追爱火葬场': {'keywords': ['追爱火葬场']},
            '宫廷侯爵': {'keywords': ['宫廷侯爵', '科举']},
            '西幻': {'keywords': ['西幻', '克苏鲁']},
            'ABO': {'keywords': ['ABO']},
            '重生': {'keywords': ['重生']},
            '幻想未来': {'keywords': ['科幻', '未来架空', '星际', '末世', '赛博朋克']},
            '架空历史': {'keywords': ['古代幻想']},
            '时尚娱乐': {'keywords': ['娱乐圈']},
            '系统快穿': {'keywords': ['系统', '快穿', '游戏网游', '全息']},
            '业界精英': {'keywords': ['职场', '商战', '业界精英']},
            '奇幻异闻': {'keywords': ['现代架空', '史诗奇幻', '东方玄幻', '仙侠修真', '灵异神怪']},
            '穿越时空': {'keywords': ['穿越时空', '穿书']},
            '年下': {'keywords': ['年下']},
            '都市情缘': {'keywords': ['都市']},
            '情有独钟': {'keywords': ['情有独钟']},
            '天赐良缘': {'keywords': ['天作之合']},
        }

    def generate_report(self):

        date = datetime.now()

        """生成数据分析报告"""
        report = "# 晋江文学城百合频道数据分析报告\n\n"
        report += "> 基于爬虫数据，分析百合文学的创作趋势\n\n"
        report += "![晋江文学城百合频道作品封面合集](jjwxc-gl-cover-collage.jpg)\n\n"

        report += "## 一、数据来源\n\n"
        report += "通过爬取晋江文学城百合频道**季度排行榜**数据，抓取字段包括：\n"
        report += "- 排名、作者、文名、类型、进度、字数\n"
        report += "- 作品视角、签约状态、发表时间、内容标签\n"
        report += "- 章节（标题、首发时间）等等\n\n"

        report += f"**分析样本**: {self.total_novels} 部百合小说作品\n\n"
        report += f"**采集时间**: {date.year}年{date.month}月{date.day}日\n\n"

        report += "## 二、频道核心数据画像\n\n"
        report += self.generate_content_analysis()

        report += "---\n\n"
        report += "*（分析工具：Python）*\n"

        # 保存报告
        with open("report.md", "w", encoding="utf-8") as f:
            f.write(report)

        print("分析报告已生成: report.md")
        return report

    def generate_content_analysis(self):
        """生成内容属性分析"""
        report = "### 1. 内容属性分布\n\n"

        # 原创性分析
        originalities = [novel['originality']
                         for novel in self.novels_data if 'originality' in novel]
        originality_counts = Counter(originalities).most_common(2)

        # 时代背景分析
        eras = [novel['era'] for novel in self.novels_data if 'era' in novel]
        era_counts = Counter(eras)
        top_eras = era_counts.most_common(3)

        # 叙事视角分析
        perspectives = [novel['perspective']
                        for novel in self.novels_data if 'perspective' in novel]
        perspective_counts = Counter(perspectives)
        top_perspectives = perspective_counts.most_common(3)

        # 风格基调分析
        style_mapping = {
            '虐文': ['虐文', '悲剧', 'BE'],
            '正剧': ['正剧', '悬疑推理', '现实'],
            '轻松': ['轻松', '甜文', '日常'],
        }

        style_counts = {style: 0 for style in style_mapping}
        for novel in self.novels_data:
            tags = novel.get('tags', [])

            # 检查每个风格类别
            for style, keywords in style_mapping.items():
                if any(keyword in tag for tag in tags for keyword in keywords):
                    style_counts[style] += 1
                    # 每本小说只计入一个主要风格
                    break

        # 创建表格
        report += "| **维度**     | **特征** | **占比** | \n"
        report += "| ------------ | -------- | -------- | \n"

        # 原创性
        for i, (originality, count) in enumerate(originality_counts):
            percent = count / self.total_novels * 100
            if i == 0:
                report += f"| **原创性**   | {originality.ljust(6)} |"
                report += f" {percent:.1f}% ".rjust(10)
                report += "|\n"
            else:
                report += f"|              | {originality.ljust(6)} |"
                report += f" {percent:.1f}% ".rjust(10)
                report += "|\n"

        # 时代背景
        for i, (era, count) in enumerate(top_eras):
            percent = count / self.total_novels * 100
            if i == 0:
                report += f"| **时代背景** | {era.ljust(4)} |"
                report += f" {percent:.1f}% ".rjust(10)
                report += "|\n"
            else:
                report += f"|              | {era.ljust(4)} |"
                report += f" {percent:.1f}% ".rjust(10)
                report += "|\n"

        # 叙事视角
        for i, (perspective, count) in enumerate(top_perspectives):
            percent = count / self.total_novels * 100
            if i == 0:
                report += f"| **叙事视角** | {perspective.ljust(6)} |"
                report += f" {percent:.1f}% ".rjust(10)
                report += "|\n"
            else:
                report += f"|              | {perspective.ljust(6)} |"
                report += f" {percent:.1f}% ".rjust(10)
                report += "|\n"

        # 风格基调
        for i, (style, count) in enumerate(
            sorted(style_counts.items(),
                   key=lambda item: item[1], reverse=True)
        ):
            percent = count / self.total_novels * 100
            if i == 0:
                report += f"| **风格基调** | {style.ljust(6)} |"
                report += f" {percent:.1f}% ".rjust(10)
                report += "|\n"
            else:
                report += f"|              | {style.ljust(6)} |"
                report += f" {percent:.1f}% ".rjust(10)
                report += "|\n"

        report += "\n\n"

        # 热门标签分析
        # 收集所有标签
        all_tags = []
        for novel in self.novels_data:
            if 'tags' in novel and isinstance(novel['tags'], list):
                all_tags.extend(novel['tags'])

        # 统计标签频率
        tag_counter = Counter(all_tags)
        top_tags = tag_counter.most_common(10)

        # 创建报告
        report += "### 2. 热门标签分析TOP10\n\n"
        report += "| **排名** | **标签**   | **出现次数** | **占比** |\n"
        report += "| -------- | ---------- | ------------ | -------- |\n"

        for rank, (tag, count) in enumerate(top_tags, 1):
            percent = count / self.total_novels * 100
            report += f"| {str(rank).rjust(8)} | {tag.encode('gbk').ljust(10).decode('gbk')} |"
            report += f"{str(count).rjust(13)} |"
            report += f" {percent:.1f}% ".rjust(10)
            report += "|\n"

        report += "\n"

        return report


def main():
    # 创建爬虫
    spider = JJWXCSpider()

    # 检查是否有已保存数据
    if not spider.load_data():
        dir = os.listdir(covers_path)
        if len(dir) != 0:
            for filename in dir:
                os.remove(f"{covers_path}/{filename}")
        # 没有数据，运行爬虫
        novels_data = spider.parse_gl_channel()

    # 进行数据分析
    analyzer = DataAnalyzer(
        spider.novels_data,
        spider.genre_mapping,
        spider.diligent_authors
    )

    # 生成封面拼贴图
    create_cover_collage()

    analyzer.generate_report()


if __name__ == "__main__":
    main()
